
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
  <head>
    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>tensorflowonspark.TFCluster &#8212; TensorFlowOnSpark 1.3.3 documentation</title>
    <link rel="stylesheet" href="../../_static/classic.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
    <script type="text/javascript" src="../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../_static/doctools.js"></script>
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" /> 
  </head><body>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../index.html">TensorFlowOnSpark 1.3.3 documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li> 
      </ul>
    </div>  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body" role="main">
            
  <h1>Source code for tensorflowonspark.TFCluster</h1><div class="highlight"><pre>
<span></span><span class="c1"># Copyright 2017 Yahoo Inc.</span>
<span class="c1"># Licensed under the terms of the Apache 2.0 license.</span>
<span class="c1"># Please see LICENSE file in the project root for terms.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">This module provides a high-level API to manage the TensorFlowOnSpark cluster.</span>

<span class="sd">There are three main phases of operation:</span>

<span class="sd">1. **Reservation/Startup** - reserves a port for the TensorFlow process on each executor, starts a multiprocessing.Manager to</span>
<span class="sd">   listen for data/control messages, and then launches the Tensorflow main function on the executors.</span>

<span class="sd">2. **Data feeding** - *For InputMode.SPARK only*. Sends RDD data to the TensorFlow nodes via each executor&#39;s multiprocessing.Manager.  PS</span>
<span class="sd">   nodes will tie up their executors, so they won&#39;t receive any subsequent data feeding tasks.</span>

<span class="sd">3. **Shutdown** - sends a shutdown control message to the multiprocessing.Managers of the PS nodes and pushes end-of-feed markers into the data</span>
<span class="sd">   queues of the worker nodes.</span>

<span class="sd">&quot;&quot;&quot;</span>

<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">division</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">nested_scopes</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">print_function</span>

<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">random</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">threading</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">from</span> <span class="nn">pyspark.streaming</span> <span class="k">import</span> <span class="n">DStream</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="k">import</span> <span class="n">reservation</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="k">import</span> <span class="n">TFManager</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="k">import</span> <span class="n">TFSparkNode</span>

<span class="c1"># status of TF background job</span>
<span class="n">tf_status</span> <span class="o">=</span> <span class="p">{}</span>


<div class="viewcode-block" id="InputMode"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.InputMode">[docs]</a><span class="k">class</span> <span class="nc">InputMode</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
  <span class="sd">&quot;&quot;&quot;Enum for the input modes of data feeding.&quot;&quot;&quot;</span>
  <span class="n">TENSORFLOW</span> <span class="o">=</span> <span class="mi">0</span>                <span class="c1">#: TensorFlow application is responsible for reading any data.</span>
  <span class="n">SPARK</span> <span class="o">=</span> <span class="mi">1</span>                     <span class="c1">#: Spark is responsible for feeding data to the TensorFlow application via an RDD.</span></div>


<div class="viewcode-block" id="TFCluster"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.TFCluster">[docs]</a><span class="k">class</span> <span class="nc">TFCluster</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

  <span class="n">sc</span> <span class="o">=</span> <span class="kc">None</span>                     <span class="c1">#: SparkContext</span>
  <span class="n">defaultFS</span> <span class="o">=</span> <span class="kc">None</span>              <span class="c1">#: Default FileSystem string, e.g. ``file://`` or ``hdfs://&lt;namenode&gt;/``</span>
  <span class="n">working_dir</span> <span class="o">=</span> <span class="kc">None</span>            <span class="c1">#: Current working directory</span>
  <span class="n">num_executors</span> <span class="o">=</span> <span class="kc">None</span>          <span class="c1">#: Number of executors in the Spark job (and therefore, the number of nodes in the TensorFlow cluster).</span>
  <span class="n">nodeRDD</span> <span class="o">=</span> <span class="kc">None</span>                <span class="c1">#: RDD representing the nodes of the cluster, i.e. ``sc.parallelize(range(num_executors), num_executors)``</span>
  <span class="n">cluster_id</span> <span class="o">=</span> <span class="kc">None</span>             <span class="c1">#: Unique ID for this cluster, used to invalidate state for new clusters.</span>
  <span class="n">cluster_info</span> <span class="o">=</span> <span class="kc">None</span>           <span class="c1">#: Cluster node reservations</span>
  <span class="n">cluster_meta</span> <span class="o">=</span> <span class="kc">None</span>           <span class="c1">#: Cluster metadata dictionary, e.g. cluster_id, defaultFS, reservation.Server address, etc.</span>
  <span class="n">input_mode</span> <span class="o">=</span> <span class="kc">None</span>             <span class="c1">#: TFCluster.InputMode for this cluster</span>
  <span class="n">queues</span> <span class="o">=</span> <span class="kc">None</span>                 <span class="c1">#: *INTERNAL_USE*</span>
  <span class="n">server</span> <span class="o">=</span> <span class="kc">None</span>                 <span class="c1">#: reservation.Server for this cluster</span>

<div class="viewcode-block" id="TFCluster.train"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.TFCluster.train">[docs]</a>  <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataRDD</span><span class="p">,</span> <span class="n">num_epochs</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">feed_timeout</span><span class="o">=</span><span class="mi">600</span><span class="p">,</span> <span class="n">qname</span><span class="o">=</span><span class="s1">&#39;input&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;*For InputMode.SPARK only*.  Feeds Spark RDD partitions into the TensorFlow worker nodes</span>

<span class="sd">    It is the responsibility of the TensorFlow &quot;main&quot; function to interpret the rows of the RDD.</span>

<span class="sd">    Since epochs are implemented via ``RDD.union()`` and the entire RDD must generally be processed in full, it is recommended</span>
<span class="sd">    to set ``num_epochs`` to closely match your training termination condition (e.g. steps or accuracy).  See ``TFNode.DataFeed``</span>
<span class="sd">    for more details.</span>

<span class="sd">    Args:</span>
<span class="sd">      :dataRDD: input data as a Spark RDD.</span>
<span class="sd">      :num_epochs: number of times to repeat the dataset during training.</span>
<span class="sd">      :feed_timeout: number of seconds after which data feeding times out (600 sec default)</span>
<span class="sd">      :qname: *INTERNAL USE*.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Feeding training data&quot;</span><span class="p">)</span>
    <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_mode</span> <span class="o">==</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">SPARK</span><span class="p">,</span> <span class="s2">&quot;TFCluster.train() requires InputMode.SPARK&quot;</span>
    <span class="k">assert</span> <span class="n">qname</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">queues</span><span class="p">,</span> <span class="s2">&quot;Unknown queue: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">qname</span><span class="p">)</span>
    <span class="k">assert</span> <span class="n">num_epochs</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;num_epochs cannot be negative&quot;</span>

    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataRDD</span><span class="p">,</span> <span class="n">DStream</span><span class="p">):</span>
      <span class="c1"># Spark Streaming</span>
      <span class="n">dataRDD</span><span class="o">.</span><span class="n">foreachRDD</span><span class="p">(</span><span class="k">lambda</span> <span class="n">rdd</span><span class="p">:</span> <span class="n">rdd</span><span class="o">.</span><span class="n">foreachPartition</span><span class="p">(</span><span class="n">TFSparkNode</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_meta</span><span class="p">,</span> <span class="n">feed_timeout</span><span class="o">=</span><span class="n">feed_timeout</span><span class="p">,</span> <span class="n">qname</span><span class="o">=</span><span class="n">qname</span><span class="p">)))</span>
    <span class="k">else</span><span class="p">:</span>
      <span class="c1"># Spark RDD</span>
      <span class="c1"># if num_epochs unspecified, pick an arbitrarily &quot;large&quot; number for now</span>
      <span class="c1"># TODO: calculate via dataRDD.count() / batch_size / max_steps</span>
      <span class="k">if</span> <span class="n">num_epochs</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="n">num_epochs</span> <span class="o">=</span> <span class="mi">10</span>
      <span class="n">rdds</span> <span class="o">=</span> <span class="p">[</span><span class="n">dataRDD</span><span class="p">]</span> <span class="o">*</span> <span class="n">num_epochs</span>
      <span class="n">unionRDD</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">rdds</span><span class="p">)</span>
      <span class="n">unionRDD</span><span class="o">.</span><span class="n">foreachPartition</span><span class="p">(</span><span class="n">TFSparkNode</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_meta</span><span class="p">,</span> <span class="n">feed_timeout</span><span class="o">=</span><span class="n">feed_timeout</span><span class="p">,</span> <span class="n">qname</span><span class="o">=</span><span class="n">qname</span><span class="p">))</span></div>

<div class="viewcode-block" id="TFCluster.inference"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.TFCluster.inference">[docs]</a>  <span class="k">def</span> <span class="nf">inference</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataRDD</span><span class="p">,</span> <span class="n">feed_timeout</span><span class="o">=</span><span class="mi">600</span><span class="p">,</span> <span class="n">qname</span><span class="o">=</span><span class="s1">&#39;input&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;*For InputMode.SPARK only*: Feeds Spark RDD partitions into the TensorFlow worker nodes and returns an RDD of results</span>

<span class="sd">    It is the responsibility of the TensorFlow &quot;main&quot; function to interpret the rows of the RDD and provide valid data for the output RDD.</span>

<span class="sd">    This will use the distributed TensorFlow cluster for inferencing, so the TensorFlow &quot;main&quot; function should be capable of inferencing.</span>
<span class="sd">    Per Spark design, the output RDD will be lazily-executed only when a Spark action is invoked on the RDD.</span>

<span class="sd">    Args:</span>
<span class="sd">      :dataRDD: input data as a Spark RDD</span>
<span class="sd">      :feed_timeout: number of seconds after which data feeding times out (600 sec default)</span>
<span class="sd">      :qname: *INTERNAL_USE*</span>

<span class="sd">    Returns:</span>
<span class="sd">      A Spark RDD representing the output of the TensorFlow inferencing</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Feeding inference data&quot;</span><span class="p">)</span>
    <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_mode</span> <span class="o">==</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">SPARK</span><span class="p">,</span> <span class="s2">&quot;TFCluster.inference() requires InputMode.SPARK&quot;</span>
    <span class="k">assert</span> <span class="n">qname</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">queues</span><span class="p">,</span> <span class="s2">&quot;Unknown queue: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">qname</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">dataRDD</span><span class="o">.</span><span class="n">mapPartitions</span><span class="p">(</span><span class="n">TFSparkNode</span><span class="o">.</span><span class="n">inference</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">,</span> <span class="n">feed_timeout</span><span class="o">=</span><span class="n">feed_timeout</span><span class="p">,</span> <span class="n">qname</span><span class="o">=</span><span class="n">qname</span><span class="p">))</span></div>

<div class="viewcode-block" id="TFCluster.shutdown"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.TFCluster.shutdown">[docs]</a>  <span class="k">def</span> <span class="nf">shutdown</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ssc</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">grace_secs</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Stops the distributed TensorFlow cluster.</span>

<span class="sd">    Args:</span>
<span class="sd">      :ssc: *For Streaming applications only*. Spark StreamingContext</span>
<span class="sd">      :grace_secs: Grace period to wait before terminating the Spark application, e.g. to allow the chief worker to perform any final/cleanup duties like exporting or evaluating the model.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Stopping TensorFlow nodes&quot;</span><span class="p">)</span>

    <span class="c1"># identify ps/workers</span>
    <span class="n">ps_list</span><span class="p">,</span> <span class="n">worker_list</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">:</span>
      <span class="p">(</span><span class="n">ps_list</span> <span class="k">if</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;job_name&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;ps&#39;</span> <span class="k">else</span> <span class="n">worker_list</span><span class="p">)</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">node</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">ssc</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
      <span class="c1"># Spark Streaming</span>
      <span class="k">while</span> <span class="ow">not</span> <span class="n">ssc</span><span class="o">.</span><span class="n">awaitTerminationOrTimeout</span><span class="p">(</span><span class="mi">1</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">done</span><span class="p">:</span>
          <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Server done, stopping StreamingContext&quot;</span><span class="p">)</span>
          <span class="n">ssc</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="n">stopSparkContext</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">stopGraceFully</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
          <span class="k">break</span>
    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_mode</span> <span class="o">==</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">TENSORFLOW</span><span class="p">:</span>
      <span class="c1"># in TENSORFLOW mode, there is no &quot;data feeding&quot; job, only a &quot;start&quot; job, so we must wait for the TensorFlow workers</span>
      <span class="c1"># to complete all tasks, while accounting for any PS tasks which run indefinitely.</span>
      <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
      <span class="k">while</span> <span class="n">count</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
        <span class="n">st</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">statusTracker</span><span class="p">()</span>
        <span class="n">jobs</span> <span class="o">=</span> <span class="n">st</span><span class="o">.</span><span class="n">getActiveJobsIds</span><span class="p">()</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">jobs</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
          <span class="k">break</span>
        <span class="n">stages</span> <span class="o">=</span> <span class="n">st</span><span class="o">.</span><span class="n">getActiveStageIds</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">stages</span><span class="p">:</span>
          <span class="n">si</span> <span class="o">=</span> <span class="n">st</span><span class="o">.</span><span class="n">getStageInfo</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
          <span class="k">if</span> <span class="n">si</span><span class="o">.</span><span class="n">numActiveTasks</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">ps_list</span><span class="p">):</span>
            <span class="c1"># if we only have PS tasks left, check that we see this condition a couple times</span>
            <span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>

    <span class="c1"># shutdown queues and managers for &quot;worker&quot; executors.</span>
    <span class="c1"># note: in SPARK mode, this job will immediately queue up behind the &quot;data feeding&quot; job.</span>
    <span class="c1"># in TENSORFLOW mode, this will only run after all workers have finished.</span>
    <span class="n">workers</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">worker_list</span><span class="p">)</span>
    <span class="n">workerRDD</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">workers</span><span class="p">),</span> <span class="n">workers</span><span class="p">)</span>
    <span class="n">workerRDD</span><span class="o">.</span><span class="n">foreachPartition</span><span class="p">(</span><span class="n">TFSparkNode</span><span class="o">.</span><span class="n">shutdown</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">queues</span><span class="p">))</span>
    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">grace_secs</span><span class="p">)</span>

    <span class="c1"># exit Spark application w/ err status if TF job had any errors</span>
    <span class="k">if</span> <span class="s1">&#39;error&#39;</span> <span class="ow">in</span> <span class="n">tf_status</span><span class="p">:</span>
      <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Exiting Spark application with error status.&quot;</span><span class="p">)</span>
      <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">cancelAllJobs</span><span class="p">()</span>
      <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
      <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>

    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Shutting down cluster&quot;</span><span class="p">)</span>
    <span class="c1"># shutdown queues and managers for &quot;PS&quot; executors.</span>
    <span class="c1"># note: we have to connect/shutdown from the spark driver, because these executors are &quot;busy&quot; and won&#39;t accept any other tasks.</span>
    <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="n">ps_list</span><span class="p">:</span>
      <span class="n">addr</span> <span class="o">=</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;addr&#39;</span><span class="p">]</span>
      <span class="n">authkey</span> <span class="o">=</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;authkey&#39;</span><span class="p">]</span>
      <span class="n">m</span> <span class="o">=</span> <span class="n">TFManager</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">addr</span><span class="p">,</span> <span class="n">authkey</span><span class="p">)</span>
      <span class="n">q</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_queue</span><span class="p">(</span><span class="s1">&#39;control&#39;</span><span class="p">)</span>
      <span class="n">q</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
      <span class="n">q</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>

    <span class="c1"># wait for all jobs to finish</span>
    <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
      <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
      <span class="n">st</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sc</span><span class="o">.</span><span class="n">statusTracker</span><span class="p">()</span>
      <span class="n">jobs</span> <span class="o">=</span> <span class="n">st</span><span class="o">.</span><span class="n">getActiveJobsIds</span><span class="p">()</span>
      <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">jobs</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">break</span></div>

<div class="viewcode-block" id="TFCluster.tensorboard_url"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.TFCluster.tensorboard_url">[docs]</a>  <span class="k">def</span> <span class="nf">tensorboard_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Utility function to get the Tensorboard URL&quot;&quot;&quot;</span>
    <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_info</span><span class="p">:</span>
      <span class="k">if</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;tb_port&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">return</span> <span class="s2">&quot;http://</span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">node</span><span class="p">[</span><span class="s1">&#39;host&#39;</span><span class="p">],</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;tb_port&#39;</span><span class="p">])</span>
    <span class="k">return</span> <span class="kc">None</span></div></div>


<div class="viewcode-block" id="run"><a class="viewcode-back" href="../../tensorflowonspark.TFCluster.html#tensorflowonspark.TFCluster.run">[docs]</a><span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">map_fun</span><span class="p">,</span> <span class="n">tf_args</span><span class="p">,</span> <span class="n">num_executors</span><span class="p">,</span> <span class="n">num_ps</span><span class="p">,</span> <span class="n">tensorboard</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">input_mode</span><span class="o">=</span><span class="n">InputMode</span><span class="o">.</span><span class="n">TENSORFLOW</span><span class="p">,</span>
        <span class="n">log_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">driver_ps_nodes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">master_node</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">reservation_timeout</span><span class="o">=</span><span class="mi">600</span><span class="p">,</span> <span class="n">queues</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;input&#39;</span><span class="p">,</span> <span class="s1">&#39;output&#39;</span><span class="p">,</span> <span class="s1">&#39;error&#39;</span><span class="p">]):</span>
  <span class="sd">&quot;&quot;&quot;Starts the TensorFlowOnSpark cluster and Runs the TensorFlow &quot;main&quot; function on the Spark executors</span>

<span class="sd">  Args:</span>
<span class="sd">    :sc: SparkContext</span>
<span class="sd">    :map_fun: user-supplied TensorFlow &quot;main&quot; function</span>
<span class="sd">    :tf_args: ``argparse`` args, or command-line ``ARGV``.  These will be passed to the ``map_fun``.</span>
<span class="sd">    :num_executors: number of Spark executors.  This should match your Spark job&#39;s ``--num_executors``.</span>
<span class="sd">    :num_ps: number of Spark executors which are reserved for TensorFlow PS nodes.  All other executors will be used as TensorFlow worker nodes.</span>
<span class="sd">    :tensorboard: boolean indicating if the chief worker should spawn a Tensorboard server.</span>
<span class="sd">    :input_mode: TFCluster.InputMode</span>
<span class="sd">    :log_dir: directory to save tensorboard event logs.  If None, defaults to a fixed path on local filesystem.</span>
<span class="sd">    :driver_ps_nodes: run the PS nodes on the driver locally instead of on the spark executors; this help maximizing computing resources (esp. GPU). You will need to set cluster_size = num_executors + num_ps</span>
<span class="sd">    :master_node: name of the &quot;master&quot; or &quot;chief&quot; node in the cluster_template, used for `tf.estimator` applications.</span>
<span class="sd">    :reservation_timeout: number of seconds after which cluster reservation times out (600 sec default)</span>
<span class="sd">    :queues: *INTERNAL_USE*</span>

<span class="sd">  Returns:</span>
<span class="sd">    A TFCluster object representing the started cluster.</span>
<span class="sd">  &quot;&quot;&quot;</span>
  <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Reserving TFSparkNodes </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">&quot;w/ TensorBoard&quot;</span> <span class="k">if</span> <span class="n">tensorboard</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
  <span class="k">assert</span> <span class="n">num_ps</span> <span class="o">&lt;</span> <span class="n">num_executors</span><span class="p">,</span> <span class="s2">&quot;num_ps cannot be greater than num_executors (i.e. num_executors == num_ps + num_workers)&quot;</span>

  <span class="k">if</span> <span class="n">driver_ps_nodes</span> <span class="ow">and</span> <span class="n">input_mode</span> <span class="o">!=</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">TENSORFLOW</span><span class="p">:</span>
    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;running PS nodes on driver locally is only supported in InputMode.TENSORFLOW&#39;</span><span class="p">)</span>

  <span class="c1"># build a cluster_spec template using worker_nums</span>
  <span class="n">cluster_template</span> <span class="o">=</span> <span class="p">{}</span>
  <span class="n">cluster_template</span><span class="p">[</span><span class="s1">&#39;ps&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_ps</span><span class="p">)</span>
  <span class="k">if</span> <span class="n">master_node</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
    <span class="n">cluster_template</span><span class="p">[</span><span class="s1">&#39;worker&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_ps</span><span class="p">,</span> <span class="n">num_executors</span><span class="p">)</span>
  <span class="k">else</span><span class="p">:</span>
    <span class="n">cluster_template</span><span class="p">[</span><span class="n">master_node</span><span class="p">]</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_ps</span><span class="p">,</span> <span class="n">num_ps</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">num_executors</span> <span class="o">&gt;</span> <span class="n">num_ps</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span>
      <span class="n">cluster_template</span><span class="p">[</span><span class="s1">&#39;worker&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_ps</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">num_executors</span><span class="p">)</span>
  <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;cluster_template: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">cluster_template</span><span class="p">))</span>

  <span class="c1"># get default filesystem from spark</span>
  <span class="n">defaultFS</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jsc</span><span class="o">.</span><span class="n">hadoopConfiguration</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;fs.defaultFS&quot;</span><span class="p">)</span>
  <span class="c1"># strip trailing &quot;root&quot; slash from &quot;file:///&quot; to be consistent w/ &quot;hdfs://...&quot;</span>
  <span class="k">if</span> <span class="n">defaultFS</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;file://&quot;</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">defaultFS</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">7</span> <span class="ow">and</span> <span class="n">defaultFS</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">):</span>
    <span class="n">defaultFS</span> <span class="o">=</span> <span class="n">defaultFS</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>

  <span class="c1"># get current working dir of spark launch</span>
  <span class="n">working_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>

  <span class="c1"># start a server to listen for reservations and broadcast cluster_spec</span>
  <span class="n">server</span> <span class="o">=</span> <span class="n">reservation</span><span class="o">.</span><span class="n">Server</span><span class="p">(</span><span class="n">num_executors</span><span class="p">)</span>
  <span class="n">server_addr</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>

  <span class="c1"># start TF nodes on all executors</span>
  <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Starting TensorFlow on executors&quot;</span><span class="p">)</span>
  <span class="n">cluster_meta</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">getrandbits</span><span class="p">(</span><span class="mi">64</span><span class="p">),</span>
    <span class="s1">&#39;cluster_template&#39;</span><span class="p">:</span> <span class="n">cluster_template</span><span class="p">,</span>
    <span class="s1">&#39;num_executors&#39;</span><span class="p">:</span> <span class="n">num_executors</span><span class="p">,</span>
    <span class="s1">&#39;default_fs&#39;</span><span class="p">:</span> <span class="n">defaultFS</span><span class="p">,</span>
    <span class="s1">&#39;working_dir&#39;</span><span class="p">:</span> <span class="n">working_dir</span><span class="p">,</span>
    <span class="s1">&#39;server_addr&#39;</span><span class="p">:</span> <span class="n">server_addr</span>
  <span class="p">}</span>
  <span class="k">if</span> <span class="n">driver_ps_nodes</span><span class="p">:</span>
    <span class="n">nodeRDD</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">num_ps</span><span class="p">,</span> <span class="n">num_executors</span><span class="p">),</span> <span class="n">num_executors</span> <span class="o">-</span> <span class="n">num_ps</span><span class="p">)</span>
  <span class="k">else</span><span class="p">:</span>
    <span class="n">nodeRDD</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">num_executors</span><span class="p">),</span> <span class="n">num_executors</span><span class="p">)</span>

  <span class="k">if</span> <span class="n">driver_ps_nodes</span><span class="p">:</span>
    <span class="k">def</span> <span class="nf">_start_ps</span><span class="p">(</span><span class="n">node_index</span><span class="p">):</span>
      <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;starting ps node locally </span><span class="si">%d</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">node_index</span><span class="p">)</span>
      <span class="n">TFSparkNode</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">map_fun</span><span class="p">,</span>
                      <span class="n">tf_args</span><span class="p">,</span>
                      <span class="n">cluster_meta</span><span class="p">,</span>
                      <span class="n">tensorboard</span><span class="p">,</span>
                      <span class="n">log_dir</span><span class="p">,</span>
                      <span class="n">queues</span><span class="p">,</span>
                      <span class="n">background</span><span class="o">=</span><span class="p">(</span><span class="n">input_mode</span> <span class="o">==</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">SPARK</span><span class="p">))([</span><span class="n">node_index</span><span class="p">])</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">cluster_template</span><span class="p">[</span><span class="s1">&#39;ps&#39;</span><span class="p">]:</span>
      <span class="n">ps_thread</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="k">lambda</span><span class="p">:</span> <span class="n">_start_ps</span><span class="p">(</span><span class="n">i</span><span class="p">))</span>
      <span class="n">ps_thread</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
      <span class="n">ps_thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>

  <span class="c1"># start TF on a background thread (on Spark driver) to allow for feeding job</span>
  <span class="k">def</span> <span class="nf">_start</span><span class="p">(</span><span class="n">status</span><span class="p">):</span>
    <span class="k">try</span><span class="p">:</span>
      <span class="n">nodeRDD</span><span class="o">.</span><span class="n">foreachPartition</span><span class="p">(</span><span class="n">TFSparkNode</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">map_fun</span><span class="p">,</span>
                                                <span class="n">tf_args</span><span class="p">,</span>
                                                <span class="n">cluster_meta</span><span class="p">,</span>
                                                <span class="n">tensorboard</span><span class="p">,</span>
                                                <span class="n">log_dir</span><span class="p">,</span>
                                                <span class="n">queues</span><span class="p">,</span>
                                                <span class="n">background</span><span class="o">=</span><span class="p">(</span><span class="n">input_mode</span> <span class="o">==</span> <span class="n">InputMode</span><span class="o">.</span><span class="n">SPARK</span><span class="p">)))</span>
    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
      <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Exception in TF background thread&quot;</span><span class="p">)</span>
      <span class="n">status</span><span class="p">[</span><span class="s1">&#39;error&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>

  <span class="n">t</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">_start</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">tf_status</span><span class="p">,))</span>
  <span class="c1"># run as daemon thread so that in spark mode main thread can exit</span>
  <span class="c1"># if feeder spark stage fails and main thread can&#39;t do explicit shutdown</span>
  <span class="n">t</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
  <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>

  <span class="c1"># wait for executors to register and start TFNodes before continuing</span>
  <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Waiting for TFSparkNodes to start&quot;</span><span class="p">)</span>
  <span class="n">cluster_info</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">await_reservations</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">tf_status</span><span class="p">,</span> <span class="n">reservation_timeout</span><span class="p">)</span>
  <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;All TFSparkNodes started&quot;</span><span class="p">)</span>

  <span class="c1"># print cluster_info and extract TensorBoard URL</span>
  <span class="n">tb_url</span> <span class="o">=</span> <span class="kc">None</span>
  <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="n">cluster_info</span><span class="p">:</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">node</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;tb_port&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
      <span class="n">tb_url</span> <span class="o">=</span> <span class="s2">&quot;http://</span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">node</span><span class="p">[</span><span class="s1">&#39;host&#39;</span><span class="p">],</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;tb_port&#39;</span><span class="p">])</span>

  <span class="k">if</span> <span class="n">tb_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;========================================================================================&quot;</span><span class="p">)</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;TensorBoard running at:       </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tb_url</span><span class="p">))</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;========================================================================================&quot;</span><span class="p">)</span>

  <span class="c1"># since our &quot;primary key&quot; for each executor&#39;s TFManager is (host, executor_id), sanity check for duplicates</span>
  <span class="c1"># Note: this may occur if Spark retries failed Python tasks on the same executor.</span>
  <span class="n">tb_nodes</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
  <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="n">cluster_info</span><span class="p">:</span>
    <span class="n">node_id</span> <span class="o">=</span> <span class="p">(</span><span class="n">node</span><span class="p">[</span><span class="s1">&#39;host&#39;</span><span class="p">],</span> <span class="n">node</span><span class="p">[</span><span class="s1">&#39;executor_id&#39;</span><span class="p">])</span>
    <span class="k">if</span> <span class="n">node_id</span> <span class="ow">in</span> <span class="n">tb_nodes</span><span class="p">:</span>
      <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">&quot;Duplicate cluster node id detected (host=</span><span class="si">{0}</span><span class="s2">, executor_id=</span><span class="si">{1}</span><span class="s2">)&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">node_id</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">node_id</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">+</span>
                      <span class="s2">&quot;Please ensure that:</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span>
                      <span class="s2">&quot;1. Number of executors &gt;= number of TensorFlow nodes</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span>
                      <span class="s2">&quot;2. Number of tasks per executors is 1</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span>
                      <span class="s2">&quot;3, TFCluster.shutdown() is successfully invoked when done.&quot;</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
      <span class="n">tb_nodes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">node_id</span><span class="p">)</span>

  <span class="c1"># create TFCluster object</span>
  <span class="n">cluster</span> <span class="o">=</span> <span class="n">TFCluster</span><span class="p">()</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">sc</span> <span class="o">=</span> <span class="n">sc</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">meta</span> <span class="o">=</span> <span class="n">cluster_meta</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">nodeRDD</span> <span class="o">=</span> <span class="n">nodeRDD</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">cluster_info</span> <span class="o">=</span> <span class="n">cluster_info</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">cluster_meta</span> <span class="o">=</span> <span class="n">cluster_meta</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">input_mode</span> <span class="o">=</span> <span class="n">input_mode</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">queues</span> <span class="o">=</span> <span class="n">queues</span>
  <span class="n">cluster</span><span class="o">.</span><span class="n">server</span> <span class="o">=</span> <span class="n">server</span>

  <span class="k">return</span> <span class="n">cluster</span></div>
</pre></div>

          </div>
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
        <div class="sphinxsidebarwrapper">
<div id="searchbox" style="display: none" role="search">
  <h3>Quick search</h3>
    <div class="searchformwrapper">
    <form class="search" action="../../search.html" method="get">
      <input type="text" name="q" />
      <input type="submit" value="Go" />
      <input type="hidden" name="check_keywords" value="yes" />
      <input type="hidden" name="area" value="default" />
    </form>
    </div>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../index.html">TensorFlowOnSpark 1.3.3 documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../index.html" >Module code</a> &#187;</li> 
      </ul>
    </div>
    <div class="footer" role="contentinfo">
        &#169; Copyright 2018, Yahoo Inc.
      Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.7.9.
    </div>
  </body>
</html>